package com.fengzhi.webcrawlers.jsoup;

import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.CountDownLatch;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.fengzhi.webcrawlers.dome1.LinkQueue;

public class MultiThreadCrawler extends Thread {
	String url = "http://www.nipic.com/design/acg/fengjin/index.html";
	String site = "http://www.nipic.com/";
	String filePath = "d:/fengzhi/crawlers/nipic/";

	private CountDownLatch latch;

	public MultiThreadCrawler(CountDownLatch latch) {
		this.latch = latch;
	}

	@Override
	public void run() {
		startCrawler();
	}

	private void startCrawler() {
		try {
			// 判断是否为空
			while (!LinkQueue.isUnVisitedURLsEmpty()) {
				System.out.println("getVisitedURLSize" + LinkQueue.getUnVisitedURL().size());
				// 从队列取出链接
				url = LinkQueue.unVisitedURLDeQueue();
				// 判断链接是否已经访问过
				if (!LinkQueue.getVisitedURL().contains(url)) {
					// 添加到已访问set集合
					LinkQueue.addVisitedURL(url);

					Document document = Jsoup.connect(url).get();

					ImageParser parser = new ImageParser();
					Set<String> urls = parser.getLinks(document);
					// 遍历链接并添加到未访问队列
					Iterator<String> iterator = urls.iterator();
					while (iterator.hasNext()) {
						LinkQueue.addUnVisitedURL(site + iterator.next());
					}

					// 下载图像
					parser.downloadImg(document, filePath);
				}
			}

			latch.countDown();
		} catch (Exception e) {
			e.printStackTrace();
		}

	}
}
